{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants_log</th>\n",
       "      <th>Plasma_glucose_concentration_log</th>\n",
       "      <th>blood_pressure_log</th>\n",
       "      <th>Triceps_skin_fold_thickness_log</th>\n",
       "      <th>serum_insulin_log</th>\n",
       "      <th>BMI_log</th>\n",
       "      <th>Diabetes_pedigree_function_log</th>\n",
       "      <th>Age_log</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.726987</td>\n",
       "      <td>0.723606</td>\n",
       "      <td>0.185978</td>\n",
       "      <td>0.641268</td>\n",
       "      <td>-0.921785</td>\n",
       "      <td>0.313496</td>\n",
       "      <td>0.683023</td>\n",
       "      <td>1.090723</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.611777</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.192353</td>\n",
       "      <td>0.201926</td>\n",
       "      <td>-0.921785</td>\n",
       "      <td>-1.048172</td>\n",
       "      <td>0.003389</td>\n",
       "      <td>0.218194</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.033509</td>\n",
       "      <td>1.139087</td>\n",
       "      <td>-0.350496</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.921785</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.754768</td>\n",
       "      <td>0.297648</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.611777</td>\n",
       "      <td>-5.470782</td>\n",
       "      <td>-0.192353</td>\n",
       "      <td>-0.362024</td>\n",
       "      <td>0.593035</td>\n",
       "      <td>-0.568138</td>\n",
       "      <td>-1.683477</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.539599</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.641268</td>\n",
       "      <td>1.121428</td>\n",
       "      <td>1.019098</td>\n",
       "      <td>1.886918</td>\n",
       "      <td>0.369875</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants_log  Plasma_glucose_concentration_log  blood_pressure_log  \\\n",
       "0       0.726987                          0.723606            0.185978   \n",
       "1      -1.611777                          0.000000           -0.192353   \n",
       "2       1.033509                          1.139087           -0.350496   \n",
       "3      -1.611777                         -5.470782           -0.192353   \n",
       "4       0.000000                          0.539599            0.000000   \n",
       "\n",
       "   Triceps_skin_fold_thickness_log  serum_insulin_log   BMI_log  \\\n",
       "0                         0.641268          -0.921785  0.313496   \n",
       "1                         0.201926          -0.921785 -1.048172   \n",
       "2                         0.000000          -0.921785  0.000000   \n",
       "3                        -0.362024           0.593035 -0.568138   \n",
       "4                         0.641268           1.121428  1.019098   \n",
       "\n",
       "   Diabetes_pedigree_function_log   Age_log  Target  \n",
       "0                        0.683023  1.090723       1  \n",
       "1                        0.003389  0.218194       0  \n",
       "2                        0.754768  0.297648       1  \n",
       "3                       -1.683477  0.000000       0  \n",
       "4                        1.886918  0.369875       1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"./data/FE_diabetes-log.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "feat_names = X_train.columns \n",
    "\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('logloss of each fold is: ', array([0.52090066, 0.5494948 , 0.4904192 , 0.4676726 , 0.52111098]))\n",
      "('cv logloss is:', 0.5099196504681179)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5099136382941524\n",
      "{'penalty': 'l1', 'C': 1}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XucFNWZ//HPM909F+4IA3J1gEAUBkQZQbxFdI1EXRRRdBNjcGMwu/KLcfPLRt38TMRNVmPc3exLs2qIiLoRja4G0XiNRtGIIIoM4AWJlxlBEAVEmUvPPL8/qqZphpnpHpianoHvO+lXV506p/rpEvrh1Kk6Ze6OiIhIS/JyHYCIiHR8ShYiIpKRkoWIiGSkZCEiIhkpWYiISEZKFiIikpGShYiIZKRkISIiGSlZiIhIRvFcB9BW+vbt6yUlJbkOQ0SkU3nllVc+dvfiTPX2m2RRUlLC8uXLcx2GiEinYmbvZVNPp6FERCQjJQsREclIyUJERDKKdMzCzKYCvwJiwDx3v67R9lnADUBlWHSTu89L294DWAM85O5zooxVRDqv2tpaKioqqKqqynUoHVZhYSGDBw8mkUjsVfvIkoWZxYCbgVOACmCZmS1y9zWNqt7bQiK4FnguqhhFZP9QUVFB9+7dKSkpwcxyHU6H4+5s2bKFiooKhg0btlf7iPI01ERgnbuvd/caYCFwZraNzWwC0B94IqL4RGQ/UVVVRZ8+fZQommFm9OnTZ596XlEmi0HAB2nrFWFZYzPM7HUzu9/MhgCYWR5wI/B/I4xPRPYjShQt29fjk+sB7oeBEncfBzwJLAjL/xF41N0rWmpsZrPNbLmZLd+8eXPEoYrI/uS8W//Cebf+JddhdBpRJotKYEja+mB2DWQD4O5b3L06XJ0HTAiXJwNzzOxd4JfAhWa22+B42P42dy9z97Li4ow3IIpIJzBp/gwmzZ+R6zBarVu3bqnlqVOn0qtXL84444wm61566aWMHz+e0aNHU1RUxPjx4xk/fjz3339/qz5zxYoV3Hbv7az9eN0+xZ6NKK+GWgaMNLNhBEnifODr6RXMbIC7bwhXpwFrAdz9G2l1ZgFl7n5FhLGKRKbhh2/pRQ/kOBJpLz/84Q/54osvuPXWW5vcfvPNNwPw7rvvcsYZZ/Daa6/t1eesWLGCJcuWcPzJJ+x1rNmKrGfh7klgDvA4QRK4z91Xm9lcM5sWVvuema02s5XA94BZUcUjItJeTj75ZLp3775Xbd9++21OPfVUJkyYwAknnMBbb70FwMKFCyktLeXwww9nypQp7Ny5k7lz57L4gUVMP/FvW90raa1I77Nw90eBRxuVXZ22fCVwZYZ93AHcEUF4IrIfuubh1az5cHvGems2BHWyGbcYPbAHP/nbMfscWzZmz57NvHnzGDFiBC+88AJz5szhiSee4JprruHZZ5+lf//+bN26laKiIq6++mqeW7aEK3/2Yw7r+6VI49pvJhIUEenstm7dyksvvcSMGbvGbJLJJADHHnssF154Ieeeey5nn312u8emZCGtdtFjFwEwf+r8HEcisqdsewANPYp7L5kcZTit4u707du3yTGM3/zmNyxdupTFixdz5JFH8uqrr7ZrbLm+dFY6oTUbtqe68CLSdnr37s2AAQN48MEHAaivr2flypUArF+/nqOPPpprr72W3r17U1lZSffu3fl8x+ftEpuShYhIGzv++OM599xzefrppxk8eDCPP/541m0XLlzILbfcwuGHH86YMWNYvHgxAJdffjljx45l7NixTJkyhdLSUk466STeXP0GZ0+Z1rkHuEVEDhQ7duxILT///PNZtSkpKaG8vHy3suHDhzeZXBYtWrRHWXFxMb9/KuiFaIBbRCQCHWmsojPQaSgREclIyUJERDJSshARkYyULEREJCMlCzRVscgBaf7pwUuyomQhItIG2nuK8gcffJAbbrhhn+POli6dFRFpY201RXkymSQeb/pnevr06QDt8iwLUM9CRKTN7csU5ccddxyXX345ZWVl3HTTTfzhD39g0qRJHHHEEXz1q19l06ZNAMybN4/vf//7APzzP/yAyy67jGOOOYbhw4enpgtpS+pZiMj+5Y9XwMZVmettfD14z2bc4uCx8LU9HtYZmbq6OpYvXw7Ap59+yrRp0zAzbrnlFm688Uauv/76Pdps2rSJF154gVWrVjFz5sxUz6OtKFkA7+b/MlzSk8xEJPfOO++81PL777/PzJkz2bhxI9XV1YwaNarJNmeddRZmxrhx46isrGyyzr5QshCR/Uu2PYCGHsVFj0QXy17q2rVravnSSy/lqquu4rTTTuOpp57iuuua/n4FBQWpZXdv85g0ZiEi0oFt27aNQYMG4e4sWLAgZ3EoWYiItLF9maK8sZ/+9KdMnz6do446iv79+7dhlK2j01AiIm2graYoX7JkyW7rM2bM2O0xqw0uvvhiILh09hf/feNuU5Snx9JWlCxE5MDUAccqOjKdhhIRkYyULEREJCMlCxERyUjJQkREMoo0WZjZVDN708zWmdkVTWyfZWabzey18HVxWH6Ima0Iy1ab2XejjFNEDjwXPXYRFz12Ua7D6DQiSxZmFgNuBr4GjAb+zsxGN1H1XncfH77mhWUbgMnuPh6YBFxhZgOjilVEZF81TFH+2muvMXnyZMaMGcO4ceO4995796jbFlOUA6xYsYLnn36uTeLPJMpLZycC69x9PYCZLQTOBNZkaujuNWmrBeh0mYh0El26dOHOO+9k5MiRfPjhh0yYMIFTTz2VXr16pepkO0V5JitWrGDJsiUcf/IJbRJ7S6L8ER4EfJC2XhGWNTbDzF43s/vNbEhDoZkNMbPXw31c7+4fRhiriEibGDVqFCNHjgRg4MCB9OvXj82bN2fd/u233+bUU09lwoQJnHDCCbz11lsALFy4kNLSUg4//HCmTJnCzp07mTt3LosfWMT0E/+21b2S1sr1TXkPA/e4e7WZXQIsAE4CcPcPgHHh6aeHzOx+d/8ovbGZzQZmAwwdOrR9IxeRDun6l6/njU/eyFivoU424xaHHnQoP5r4o1bH8vLLL1NTU8OIESOybjN79mzmzZvHiBEjeOGFF5gzZw5PPPEE11xzDc8++yz9+/dn69atFBUVcfXVV/PcsiVc+bMf73YHdxSiTBaVwJC09cFhWYq7b0lbnQf8ovFO3P1DMysHjgfub7TtNuA2gLKysrafZlFEZC9t2LCBb37zmyxYsIC8vOxO4mzdupWXXnppt+k9kskkAMceeywXXngh5557LmeffXYkMbckymSxDBhpZsMIksT5wNfTK5jZAHffEK5OA9aG5YOBLe6+08x6A8cB/xFhrCKyn8i2B9DQo5g/dX6bx7B9+3ZOP/10fvazn3H00Udn3c7d6du3b5NjGL/5zW9YunQpixcv5sgjj+TVV19ty5AzimzMwt2TwBzgcYIkcJ+7rzazuWY2Laz2vfDS2JXA94BZYflhwNKw/M/AL909i0dfiYjkVk1NDdOnT+fCCy/knHPOaVXb3r17M2DAgNRjUevr61m5ciUA69ev5+ijj+baa6+ld+/eVFZW0r17dz7f8Xmbf4emRHqVkbs/6u6j3H2Eu/8sLLva3ReFy1e6+xh3P9zdp7j7G2H5k+4+LiwfF55uEhHp8O677z6ee+457rjjjtQlsa252mnhwoXccsstHH744YwZM4bFixcDcPnllzN27FjGjh3LlClTKC0t5aSTTuLN1W9w9pRp+/0At4jIfqFhWvALLriACy64IKs2TU1RPnz48Caff7Fo0aI9yoqLi/n9U0EvpDMPcIuIdFhRjFXsz3Szm4iIZKRkISIiGSlZiIhIRkoWIiKSkZKFiByQ3vvmhbz3zQtzHUanoWQhItIG2nuK8gcffJAbbrihzeLPRJfOioi0obacojyZTBKPN/0zPX36dADWfryujb9B05QsRETa0KhRo1LL6VOUpyeLlhx33HEcddRRPP/881xwwQUMGzaMn//859TU1FBcXMzdd99Nv379mDdvHuXl5Vzy4zn88z/8gOEDS1i2bBkbN27kxhtvTCWTtqJkISL7lY0//znVazNPUV71RlAnm3GLgsMO5eCrrmp1LHszRTlAXV0dy5cvB+DTTz9l2rRpmBm33HILN954I9dff/0ebTZt2sQLL7zAqlWrmDlzppKFiEhnsDdTlDc477zzUsvvv/8+M2fOZOPGjVRXV+/Wc0l31llnYWaMGzeOysrKJuvsCyULEdmvZNsDaOhRHHLXnW0ew95OUd6ga9euqeVLL72Uq666itNOO42nnnqK6667rsk2BQUFqWX3tn+8j66GEhFpQ/syRXlTtm3bxqBBg3B3FixY0AYR7h0lCxGRNrSvU5Q39tOf/pTp06dz1FFH0b9//zaMtHV0GkpEpA201RTlS5Ys2W19xowZuz1mtcHFF18MBJfO/uK/b9xtivKGWNqSkoWIHJCiGKvYn+k0lIiIZKRkISL7hSiuANqf7OvxUbIQkU6vsLCQLVu2KGE0w93ZsmULhYWFe70PjVmISKc3ePBgKioq2Lx5c65DaXcbdoTfeXNti/UKCwsZPHjwXn+OkoWIdHqJRIJhw4blOoycmDX/xwAsveiBSD9Hp6FERCQjJQsREclIyUJERDKKNFmY2VQze9PM1pnZFU1sn2Vmm83stfB1cVg+3sz+Ymarzex1Mztvz72LiEh7iWyA28xiwM3AKUAFsMzMFrn7mkZV73X3OY3KvgAudPe3zWwg8IqZPe7uW6OKV0REmhdlz2IisM7d17t7DbAQODObhu7+lru/HS5/CGwCiiOLVEREWhRlshgEfJC2XhGWNTYjPNV0v5kNabzRzCYC+cA70YQpIiKZ5HqA+2GgxN3HAU8Cu03WbmYDgLuAi9y9vnFjM5ttZsvNbPmBeDOOiEh7iTJZVALpPYXBYVmKu29x9+pwdR4woWGbmfUAHgH+xd1fauoD3P02dy9z97LiYp2lEhGJSpTJYhkw0syGmVk+cD6wKL1C2HNoMA1YG5bnAw8Cd7r7/RHGKCIiWYjsaih3T5rZHOBxIAbc7u6rzWwusNzdFwHfM7NpQBL4BJgVNp8JnAD0MbOGslnuvvePmxIRkb0W6dxQ7v4o8GijsqvTlq8Ermyi3d3A3VHGJiIi2cv1ALeIiHQCShYiIpKRkoWIiGSkZCEiIhkpWYiISEZKFiIikpGShYiIZKRkISIiGSlZiIhIRkoWIiKSkZKFiIhkpGQhIiIZtTpZmFle+KwJERE5QGSVLMzsd2bWw8y6AuXAGjP7YbShtY+q2jo+Xj+DrZVT+PWz63j+7c18+nlNrsMSEelQsp2ifLS7bzezbwB/BK4AXgFuiCyydrK9qpZ44Sckq/rwi8feTJUP6lXE2EE9GTu4J2MG9mDsoJ706VaQw0hFRHIn22SRMLMEcBZwk7vXmplHGFe76de9kF4DnwHgifMWsvrDbayqDF7lldt4bPXGVN2BPQspHdST0kE9GRu+F3dXAhGR/V+2yeJW4F1gJfCcmR0CbI8qqFzp2SXBMV/qyzFf6psq215Vy+rK7ZQ3JJAPt/Hk2o/wMFX271GQShwN7/17FOboG4iIRCOrZOHu/wX8V1rRe2Y2JZqQOpYehQkmj+jD5BF9UmU7qpOsrtxG+Ye7ksjTb2xKJZDi7rsSSOnAHowd3JODexRiZjn6FiIi+yarZGFmlwHzgc+AecARBOMWT0QXWsfVrSDOpOF9mDR8VwL5vDrJ2g3bU6ewVldu59k3N1EfJpC+3fIZM3BX72Ps4J4M7KkEIiKdQ7anof7e3X9lZqcCvYFvAndxgCaLpnQtiFNWchBlJQelynbW1LFmQ9D7aOiBLFn3MXVhBjmoa35q8LwhiQzuXaQEIiIdTrbJouHX6zTgLndfbfpFy6goP8aEQ3oz4ZDeqbKq2jrWbtgenMKqCBLIbc+tJxkmkF5dEpQO7MmYQbuSyNCDuiiBiEhOZZssXjGzJ4BhwJVm1h2ojy6s/VdhIsYRQ3tzxNBdCaQ6WcebGz9LXYFVXrmd25f8ldq6IIF0L4xTOjA4ddUwDlLSpyt5eUogItI+sk0W3wbGA+vd/Qsz6wNcFF1YB5aCeIxxg3sxbnCvVFlNsp63PkpPINu448V3qUkGObp7QZzR4Smshst5h/dVAhGRaGR7NVS9mQ0Gvh6eDvmzuz8caWQHuPx4XioJNKitq+ftj3akxj9WVW7jrpfeozpMIF3zY4xpdApreHE3YkogIrKPsr0a6jrgKOB/wqLvmdlkd78qssjaSW1dLUm2A8aLH75Il3gXiuJFdEmE7+F6RxgzSMTyGD2wB6MH9mDmUUMASNbVs27zDlZVbGP1h8HVWAtf/oD5te8CUJSI7dYDGTuoJyOKuxKPaQ5JEcletqehTgPGu3s9gJktAF4FWkwWZjYV+BUQA+a5+3WNts8imDKkMiy6yd3nhdseA44Glrj7GVnG2WrbarZRkxfcpX3Jk5c0WccwCuOFdIl32SOJdEl0aXY5VS9R1GR5YbyQPNu3H+14LI9DD+7BoQf34NywrK7eeWfzrh5IeeU27lv+AXe8+C4AhYk8DhsQJpCBQRIZ2b8bCSUQEWlGtskCoBfwSbjcs6WKAGYWA24GTgEqgGVmtsjd1zSqeq+7z2liFzcAXYCmf8HbSK+CXhTWlwDObadfyxe1X7AzuZMvkl80u7yzNnj/PPk5H1d9nNrW8GqNlhJPw7bmlptLSoXxQkb1786o/t05+8jBQJBA/vrx57slkP9dUcmdf3kPCE57HTagR3ATYdgLGdW/O/nxAyeBuDvJ+iS19bXU1tdSnaxlZ20NO5O1VNXWBOvJGqqSNVQnk1Qnq6mqq6WmLkl1sobqulpq6mqpDd9r6pLU1Nfyxc4E4PzD4l/i4Z2bjqeWwcMbOj383654PHyHPds1uS1omNpLUGfXtSje6LNIax38P608VS+tlqdHuGvfuyJI31dDya62u9cNK6W1c2BHVRKAk++6OON/s13fq/m1LBtl1TbaOY7Sj0/2dlTVgkc/a0S2yeLfgFfN7BmCy2hPILgpryUTgXXuvh7AzBYCZwKNk0WT3P1pMzsxy/j2WjwvTh75ABzR74h93l+911OVrGox2TSVeKJIQs0mm55FlPbpQtmRhdTUJNj6ufHxZ7Dh03oWvV3HwvIYXl9AggKG9TmI0QcXM25QMeMHF3PYgKb/neDu1Hld6sc2WZ9MvWrqaqhK1lKVDN531tZQXRf84FYlg+WaZJKqZPBDW11XE/zQ1gU/2g0/wLX1yfCVvv9akh4s13mSuoZ3T1LvddSRpN6T1FNHvdfhBMueeiWBOrA6sIgu8OsavC3ZktUf/Yzc00+JNlr2ZsqbWnbLWM/22Nb4PdyPEX52Q7uW9pu+1yb2F68DYEvNDvbevpw2zuUp58bHPQvxWjwZ/T/qsh3gvsfMniUYtwD4kbtvbKEJwCDgg7T1CmBSE/VmmNkJwFvA5e7+QRN1Oo08ywt6CIkuUNR2+80mCTWVeBovt5iEEkD/oDvXoBKo/ByefAt4C7w+H+rjYM7hd0xK/ehiybb7si3w+hgQA8/DPQ6ehxEDj2HEMWIYMfKIYRYnjwR5VkSMGPkWJ2Zx8ixOzGLE8hLEw+V4XpxEuB6PJUiE64nYrvf8WD75sTj5sQT5sQQF4XphPFguiCcojOdTGE+QH0/QJZFPYTyfrz90GRg8fO6t5OUZRh5Y8EOZZ8HPZXAVW7hshmGYBdtJbScsT/vJDts3DKl1hLG1fTVp/gwAll70QI4j6RwmzZ/RunNEe6nFjzCzIxsVVYTvA81soLuv2MfPfxi4x92rzewSYAFwUraNzWw2MBtg6NCh+xhKx9YeSai5BPNF7Rds2L6N9z7dyobt21j10Xrc8yjuVkzcEqkf3kRe8GOcyIsTD9cTsd3f9/zBTVAQ/tjmx+MUxfIpSORTEItTEM+nS34BBbE4RfF8ChMJErE8EvE88mN5xPOMWJ51+B/IWLwWgAE9M569FemwMuWjG1vY5rT8w14JDElbH8yugexgB+5b0lbnAb/IEM/uAbjfBtwGUFZWtl9Mmd7e9iYJNfzL75mLfhVhZCLSkbSYLNx9X2aWXQaMNLNhBEnifODr6RXMbIC7bwhXpwFr9+Hz9tohte/k4mNFRDqNbO+zOLuJ4m3AKnff1FQbd0+a2RzgcYJLZ28P55SaCyx390UE92tMA5IEV1rNSvvM54FDgW5mVgF8290fz/6riYhIW2nNdB+TgWfC9RMJHqs6zMzmuvtdTTVy90eBRxuVXZ22fCVwZTNtj88yNhERiVi2ySIOHObuHwGYWX/gToKrm54jmK5cRET2U9lenDukIVGENoVlnwC1bR+WiIh0JNn2LJ41s8XA78P1c8KyrsDWSCITEZEOI9tkcSlwNnBcuL4AeMCD+/cPiGdxi4gcyLK9g9vNbAlQQ3B/xcu+a2IbERHZz2U1ZmFmM4GXCU4/zQSWmtk5UQYmIiIdR7anof4FOKrhngozKwaeAu6PKjAREek4sr0aKq/RzXdbWtFWREQ6uWx7Fo+Z2ePAPeH6eTS62U5ERPZf2Q5w/9DMZgDHhkW3ufuD0YUlIiIdSdazoLv7A4AmmBcROQBlep7FZzT9lD8juKK2RyRRiYhIh5JpivLu7RWIiIh0XLqiSUREMmqHJ7d2fLPuqwkWZuc2DhGRjko9CxERyUjJQkREMlKyEBGRjJQskjUcPXYzJQN3QNW2XEcjItIhKVns2IgZjB6+DW48DBZfDh+tyXVUIiIdipJFr6H85fVilrxaDGOmw6v/A/89Ge44A9b8AeqSuY5QRCTnlCxC2z/Ph7Nuhn9aC39zDXz6Htx3IfxqHDx3A+zYnOsQRURyRsmisa594Ljvw2Wvwfm/g74j4U//Cv8xGv73Eqh4JdcRioi0O92U15y8GBx6evDa/CYsmwev/Q5eXwgDj4SJs4PTVonCXEcqIhI59SyyUfxlOO2G4BTVab+Emh3w0HeD3sZT18C2ilxHKCISqUiThZlNNbM3zWydmV3RxPZZZrbZzF4LXxenbfuWmb0dvr4VZZxZK+wBE78Dl74MF/4Bhk6GF/4T/nMs3HsB/PU58KYm6RUR6dwiOw1lZjHgZuAUoAJYZmaL3L3xdan3uvucRm0PAn4ClBFMkf5K2PbTqOJtFTMYfmLw2vo+LPstrLgT1j4MxYcGCWXc+VDQLbdxioi0kSh7FhOBde6+3t1rgIXAmVm2PRV40t0/CRPEk8DUiOLcN72GwinXwD+tgTN/DfECeOQH8O+HwR9/BB+/nesIRUT2WZTJYhDwQdp6RVjW2Awze93M7jezIa1s23EkiuCIb8DsP8O3n4JRU4Mex01lcNd0ePOPUF+X6yjbxCG173BI7Tu5DkNE2lGur4Z6GLjH3avN7BJgAXBSto3NbDbhxOJDhw6NJsLWMoMhRwWvU38Gr9wBy2+He86HXofAURfDERdAl4NyHele05TurfOT/1kbLFyU2zg6Cx2v1mmv4xVlz6ISGJK2PjgsS3H3Le5eHa7OAyZk2zZsf5u7l7l7WXFxcZsF3ma69YOv/DN8fxWcewf0HAxP/r/gFNUf5sCGlbmOUEQkK1Emi2XASDMbZmb5wPnAovQKZjYgbXUaEKZIHge+ama9zaw38NWwrHOKJYJ7Mi56FL77Ahx+PpQ/ALeeAL89FVbdD8maXEcpItKsyE5DuXvSzOYQ/MjHgNvdfbWZzQWWu/si4HtmNg1IAp8As8K2n5jZtQQJB2Cuu38SVazt6uBS+Ntfwd/8NLjJ7+XfwAPfhm79YcJFMGEW9BiQYSciIu0r0jELd38UeLRR2dVpy1cCVzbT9nbg9ijjy6mi3jD5Upj0D7DuKXj5NvjzdfD8L2H0mcEd4kMmBWMgIiI5lusBbsnLg1FfDV5b3gmuoHr17uA01cFjg6RReg7kd8l1pCJyADvgp/uor6lh4EdO30+cT373O3a+/jr11dWZG0ahzwiY+nP4wVo44z+hvh4W/Z9gQPyJH8Mnf81NXCJywDvgexZ1W7dSlwddquCjudcGhfE4BaNGUjSmlMIxYygsLaVw1EgsP799gsrvCmXh+MV7LwanqP7ya3jxJhh1anCH+PCTgl6JiEg7OOCTRaJfPz4qNnDnlDufYufq1VSVr6aqvJztTzzB1t//HgBLJCj48pcpLB1DUWkphaWlFIwYgSUS0QVnBiXHBq9tlfDK/OC+jbsfgz5fgqO+A+P/Dgp7RheDiAhKFruYkRg0iMSgQfT46lcBcHdqKyqoKi9nZ3k5VavXsH3xI2xdeG/QpKCAgkO/HPRASkspLB1DwfDhWDyCw9pzEJz0Yzjhh8ET/F6+DR77ETw9N7gUd+J3oN9hbf+5IiIoWbTIzMgfMoT8IUPo8bWvAeD19dS+/z47w95HVXk52x56iE9/97ugTVERhYcdtlsPJL+kBGurU0bxAhg3M3hVrgies/Hq3bD8t1ByfDAg/uXTIKb/tCLSdvSL0kqWl0d+SQn5JSX0PON0ALyujpr33tvVAylfzdb7fs+nd94FQF6XLhSOHh32PkopKh1DYujQfU8gg46EQb+GU66FV+8MrqS675vQY3Aw5nHkt6BbB7yzXUQ6HSULoNqK9qm9xWIUDB9OwfDh9Jw2DQBPJqlevz41/lG1ejWf3nMPHl5plde9O4VjxlBUOiY1iJ4YPBjbm/squvaB4y6HY74Hbz0GS2+FP10Lf74expwNk2bDoAmZ9yMi0gwli4hYPE7hqFEUjhoFZ08HwGtrqX7nnd16IFsW3Am1tQDk9exJUcPVV+FprPiAAdknkMaPgn35N7DynuBRsIMm7HoUbLwgqq8tIvspJYt2ZIkEhYceSuGhh9LrnHOA4D6P6rfeTvU+dq4uZ8vtt0MyCUDsoIPCnseuMZB4v36ZE0jxl+H0X8LJV8PKhcGA+IOXwOP/AhO+BWV/H0xsKCKSBSWLHMvLz6eoNDgd1aC+uprqN99M9T6qysvZ8uKLUBc8DyNW3He3K7CKxowh3tysu4U9gtNQE78D658NehtL/gOW/CccehpMvARKjtP35uedAAANrUlEQVS0IiLSIiWLDiivoICiceMoGjcuVVa/cydVb7yRSh47V5ez489/Tj3zO96/f2rwvLA0uJkwflDaMzPMYMSU4PXpe8EzNlYsCB8Fe1j4KNjz9ChYEWmSkkUnkVdURJcjjqDLEUekyuo//5yqtWtT94BUlZez4+mnU9sTAwemEkdDDyTWqxf0PiR4FOyJVwRzUC29FR75J3jqpzD+G8EDmvp+KQffUkQ6KiWLTiyva1e6lJXRpawsVVb32WdUrVkbjoGUs7N8NZ898URqe2LIkF3jH2NKKRxzJrHx34CKZcG4xrJ5sPS/YcTJwYD4yFOCgXMROaApWexnYt2703XSRLpOmpgqq9u2jao1a3aNgby+is/++Fhqe35JSdj7OI6ir8ykoGoZsfK74J7zoHdJ0NMY/41O/ShYEdk3ShYHgFjPnnSdPJmukyenypKffhokjtXBZbxfrFjB9kceCTaakT9sBEWDj6Bw8/sUvn0NhU/8jLwjzw16GyJywFGyOEDFe/em2/HH0e3441JlyY8/Di7fDXsgO8pXsW3zJ0AxGBQ89jiFvRcxrkc91Z7Htn/9enAJr+UFM+Ca7Xrl5WGWBxjkhXXCemaktWmokxfsq6Esbdka6pK2vWH/ebb7vsJ6lpcHFtsVV8O+UvtPbxPb9Rmp/cV2/+yGfaV/fkO5WVC/oU3e7nUTVg9A3ZYN7f8fuhNKWD0O1H2yMdehdApxqwdvh8+J/iOks4j37Uu3r3yFbl/5Sqqs9qNNVK0OksfO119lx8rXqPtsJwAfrn01V6F2KoMJLkt+69iTchxJ55A6XsdMyXEkncMQjESv2sg/R8kC6JKvAdzmJPr3I9H/JLqfFPzQuTtPnDSavHo44fbFwaW77rh7+K8bT5XtVu71wcOc6uvB66C+Hvf6XeVeB/W+q7w+3Ja27PV14brvXu67r+8qb1QXT9tXw/4bynxXG/dGn++7yhu2kRZr6vumfV7qu9ez6YXgCrV+k/Xjl41NLz0DQL+jdbyysemlZ/iiHe6TUrKQVjEz6mJGXQwKRozIdTidwtK/GQ3AYXN/m+NIOgcdr9ZpOF7jMtTbV3rUmoiIZKRkISIiGSlZiIhIRkoWIiKSkQa4gYX/J5jx9dQcxyEi0lFF2rMws6lm9qaZrTOzK1qoN8PM3MzKwvV8M5tvZqvMbKWZnRhlnCIi0rLIehZmFgNuBk4BKoBlZrbI3dc0qtcduAxYmlb8HQB3H2tm/YA/mtlR7l4fVbwiItK8KHsWE4F17r7e3WuAhcCZTdS7FrgeqEorGw38CcDdNwFbgbI9m4qISHuIMlkMAj5IW68Iy1LM7EhgiLs/0qjtSmCamcXNbBgwARgSYawiItKCnA1wWzDL3L8Ds5rYfDtwGLAceA94EahrYh+zgdkAQ4cOjSpUEZEDXpQ9i0p27w0MDssadAdKgWfN7F3gaGCRmZW5e9LdL3f38e5+JtALeKvxB7j7be5e5u5lxc09g1pERPZZlMliGTDSzIaZWT5wPrCoYaO7b3P3vu5e4u4lwEvANHdfbmZdzKwrgJmdAiQbD4yLiEj7iew0lLsnzWwO8DgQA25399VmNhdY7u6LWmjeD3jczOoJeiPfjCpOERHJLNIxC3d/FHi0UdnVzdQ9MW35XeDLUcYmIiLZ03QfIiKSkZKFiIhkpLmhgPlT5+c6BBGRDk09CxERyUjJQkREMlKyEBGRjDRmIa12zTcOA+C0HMfRWeh4tY6OV+u01/FSz0JERDJSshARkYyULEREJCMlCxERyUjJQkREMlKyEBGRjJQsREQkIyULERHJSMlCREQyUrIQEZGMlCxERCQjJQsREclIyUJERDJSshARkYyULEREJCM9z0JabfSAHrkOQUTamZKFtNr8qfNzHYKItDNz91zH0CbKysp8+fLluQ5DRKRTMbNX3L0sU71IxyzMbKqZvWlm68zsihbqzTAzN7OycD1hZgvMbJWZrTWzK6OMU0REWhZZsjCzGHAz8DVgNPB3Zja6iXrdgcuApWnF5wIF7j4WmABcYmYlUcUqIiIti7JnMRFY5+7r3b0GWAic2US9a4Hrgaq0Mge6mlkcKAJqgO0RxioiIi2IMlkMAj5IW68Iy1LM7EhgiLs/0qjt/cDnwAbgfeCX7v5JhLGKiEgLcnafhZnlAf8O/KCJzROBOmAgMAz4gZkNb2Ifs81suZkt37x5c6TxiogcyKJMFpXAkLT1wWFZg+5AKfCsmb0LHA0sCge5vw485u617r4JeAHYY7Te3W9z9zJ3LysuLo7oa4iISJTJYhkw0syGmVk+cD6wqGGju29z977uXuLuJcBLwDR3X05w6ukkADPrSpBI3ogwVhERaUFkycLdk8Ac4HFgLXCfu682s7lmNi1D85uBbma2miDpzHf316OKVUREWqab8kREDmDZ3pS33yQLM9sMvLcPu+gLfNxG4bQlxdU6iqt1FFfr7I9xHeLuGQd995tksa/MbHk22bW9Ka7WUVyto7ha50COS1OUi4hIRkoWIiKSkZLFLrflOoBmKK7WUVyto7ha54CNS2MWIiKSkXoWIiKS0QGbLMzsXDNbbWb1Dc/RaKZeVs/kaMO4DjKzJ83s7fC9dzP16szstfC1qKk6bRRPi9/fzArM7N5w+9L2mEo+i5hmmdnmtONzcdQxhZ97u5ltMrPyZrabmf1XGPfr4USaHSGuE81sW9rxurqd4hpiZs+Y2Zrw7+JlTdRp92OWZVztfszMrNDMXjazlWFc1zRRJ7q/j+5+QL6Aw4AvA88CZc3UiQHvAMOBfGAlMDriuH4BXBEuXwFc30y9He1wjDJ+f+AfgVvC5fOBeztATLOAm3LwZ+oE4EigvJntpwF/BIxgCpulHSSuE4HFOTheA4Ajw+XuwFtN/Lds92OWZVztfszCY9AtXE4QPAPo6EZ1Ivv7eMD2LNx9rbu/maFats/kaEtnAgvC5QXAWRF/Xkuy+f7p8d4PnGxmluOYcsLdnwNamkr/TOBOD7wE9DKzAR0grpxw9w3uviJc/oxgWqBBjaq1+zHLMq52Fx6DHeFqInw1HnSO7O/jAZssspTxmRwR6O/uG8LljUD/ZuoVhtOzv2RmUSWUbL5/qo4H84FtA/pEFE+2MQHMCE9b3G9mQ5rYngu5+POUrcnh6Y0/mtmY9v7w8HTJEez+xEzI8TFrIS7IwTEzs5iZvQZsAp5092aPV1v/fYy3xU46KjN7Cji4iU3/4u5/aO94GrQUV/qKu7uZNXe52iHuXmnBcz7+ZGar3P2dto61k3oYuMfdq83sEoJ/aZ2U45g6shUEf552mNlpwEPAyPb6cDPrBjwAfN/dO8wTMTPElZNj5u51wHgz6wU8aGal7t7kWFRb26+Thbv/zT7uItMzOfZKS3GZ2UdmNsDdN4Td7U3N7KMyfF9vZs8S/OunrZNFNt+/oU6FBY/B7QlsaeM4WhWTu6d//jyCcaCOIJI/T/sq/YfQ3R81s1+bWV93j3wOJDNLEPwg/4+7/28TVXJyzDLFlctjFn7mVjN7BpgKpCeLyP4+6jRUy1p8JkdEFgHfCpe/BezRAzKz3mZWEC73BY4F1kQQSzbfPz3ec4A/eTi6FpGMMTU6pz2N4JxzR7AIuDC8wudoYFvaKcecMbODG85rm9lEgt+FKBN+w+ca8Ftgrbv/ezPV2v2YZRNXLo6ZmRWHPQrMrAg4hT2f8xPd38f2HM3vSC9gOsH5z2rgI+DxsHwg8GhavdMIroZ4h+D0VdRx9QGeBt4GngIOCsvLgHnh8jHAKoIrgVYB344wnj2+PzCX4EFVAIXA74F1wMvA8HY4Rpli+jdgdXh8ngEObac/U/cQPDe+Nvyz9W3gu8B3w+1G8KyWd8L/bk1ehZeDuOakHa+XgGPaKa7jCAZoXwdeC1+n5fqYZRlXux8zYBzwahhXOXB1WN4ufx91B7eIiGSk01AiIpKRkoWIiGSkZCEiIhkpWYiISEZKFiIikpGShUgrmNmOzLVabH9/eNc9ZtbNzG41s3fM7BUze9bMJplZvpk9F95UJdIhKFmItJNw/qCYu68Pi+YRTPA30t0nABcBfT2YIPFp4LzcRCqyJyULkb0Q3lF8g5mVm9kqMzsvLM8Lp354w4LnkTxqZueEzb5BeEe+mY0AJgE/dvd6AHf/q7s/EtZ9KKwv0iGomyuyd84GxgOHA32BZWb2HMHUKyXAaKAfwVQjt4dtjiW4mxpgDPCaBxPDNaUcOCqSyEX2gnoWInvnOIKZbevc/SPgzwQ/7scBv3f3enffSDDdSIMBwOZsdh4mkRoz697GcYvsFSULkfazk2DuHgjmFTrczGIt1C8AqiKPSiQLShYie+d54LzwYTTFBI8ufRl4geDBS3lm1p/g8ZsN1gJfAvDg2SPLgWvSZi8tMbPTw+U+wMfuXtteX0ikJUoWInvnQYLZP1cCfwL+OTzt9ADBzK5rgLsJHpKzLWzzCLsnj4sJnoS4zszKgTvY9fySKWF9kQ5Bs86KtDEz6+bBE9T6EPQ2jnX3jeEzCJ4J15sb2G7Yx/8CV7j7W+0QskhGuhpKpO0tDh9Skw9cG/Y4cPedZvYTguckv99c4/ChTg8pUUhHop6FiIhkpDELERHJSMlCREQyUrIQEZGMlCxERCQjJQsREclIyUJERDL6//xGa/vzfDkvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle\n",
    "\n",
    "cPickle.dump(grid.best_estimator_, open(\"L2_log_logloss.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
